#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Time    : 2018/1/28 0028 15:59
# @Author  : Arliki
# @File    : comment_url.py

import json
import re
import scrapy
import time
import redis
import chardet
from scrapy_redis.spiders import RedisSpider

end_cursor = 0
commentid = 0
vid_url = "https://ncgi.video.qq.com/fcgi-bin/video_comment_id?otype=json&op=3&vid=%s&_=1517122173931"
comment_urls = "https://video.coral.qq.com/varticle/%s/comment/v2?callback=_varticle%scommentv2&orinum=30&oriorder=o&pageflag=1&cursor=%s&scorecursor=0&orirepnum=2&reporder=o&reppageflag=1&source=9&_=1517212747602"
class CommentSpider(RedisSpider):
    name = 'comment_url'
    redis_key = 'qqlive:start_url'
    custom_settings = {
        'REDIS_PARAMS': {
            'db': 3,
            'password': '5tgbnhy67ujm'
        }
    }
    def __init__(self, *args, **kwargs):
        domain = kwargs.pop('domain', '')
        self.allowed_domains = filter(None, domain.split(','))
        super(CommentSpider, self).__init__(*args, **kwargs)

    def parse(self, response):
        global vid_url
        source = response.body.decode('utf-8')
        vid = re.search('&vid=(.*?)&', source, re.S).group(1)
        com_id_url = vid_url % vid
        yield scrapy.Request(com_id_url, self.comment_id, dont_filter=True)

    def comment_id(self, response):
        global commentid, end_cursor, comment_urls
        commentid = re.search('"comment_id":"(.*?)"', response.body.decode('utf-8'), re.S).group(1)
        url = comment_urls % (commentid, commentid, end_cursor)
        yield scrapy.Request(url, self.comment_url, dont_filter=True)

    def comment_url(self, response):
        global commentid, end_cursor, comment_urls
        jsdic = re.search('.*?\((.*?)\)$', response.body.decode('utf-8')).group(1)
        jsdic = json.loads(jsdic, encoding='utf-8')
        jsdata = jsdic['data']
        end_cursor = int(jsdata['last'])
        r = redis.Redis(db='3', password='5tgbnhy67ujm')
        r.lpush('qqlive:comment_url', response.url)
        next_url = comment_urls % (commentid, commentid, end_cursor)
        if jsdata['hasnext']:
            yield scrapy.Request(next_url, self.comment_url, dont_filter=True)

# lpush qqlive:start_url https://v.qq.com/x/cover/2tr6sh7iw7wyvhl.html
